import * as cheerio from 'cheerio'
import { convert } from 'html-to-text'

import axios from 'axios'
import iconv from 'iconv-lite'

// cheerio
export const htmlToCheerio = html => {
  return cheerio.load(html, {
    decodeEntities: false,
  })
}

export function htmlToText(html, wordwrap = false) {
  return convert(html, {
    wordwrap, // 设置为 null 或 false 可禁用换行
  })
}

// 请求网页
export async function crawlWeb(url, options) {
  const res = await axios.get(url, {
    responseType: 'arraybuffer',
    ...options,
  })
  const html = iconv.decode(res.data, 'utf-8')
  // const html = res.data
  return {
    html,
    $: htmlToCheerio(html),
  }
}
